#!/usr/bin/env python
#coding=utf-8

"""
copyright (c) 2009, paketka@gmail.com et. al
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

	* Redistributions of source code must retain the above copyright notice,
	  this list of conditions and the following disclaimer.

	* Redistributions in binary form must reproduce the above copyright notice,
	  this list of conditions and the following disclaimer in the documentation
	  and/or other materials provided with the distribution.

	* Neither the name of the <ORGANIZATION> nor the names of its contributors
	  may be used to endorse or promote products derived from this software
 	  without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

from django.core.management import setup_environ
import settings
setup_environ(settings)
from django.core.files import File
from django.core.exceptions import ObjectDoesNotExist
from django.db import IntegrityError

import sys
import Image
import httplib
import os.path
import utils.fetcher
import unicodedata
import re
import datetime
from psp.models import *

def fixNames():
	for mp in Person.objects.all():
		if not mp.name.istitle():
			old = '%s %s' % (mp.name, mp.surname)
			mp.title += ' ' + mp.name
			list = mp.surname.split()
			mp.name = list.pop(0)
			newSurname = ''
			for n in list:
				newSurname += n + ' '
			mp.surname = newSurname.strip()
			mp.save()
			print 'Fixed: %s to %s %s' % (old, mp.name, mp.surname)


def fixOrgs():
	"""
	fixes group names. turns origName into name and find proper
	parent group for newly created subgroup.
	"""
	splitPattern = re.compile(u'(?P<boardName>.*)(?P<subBoardName>[Pp]odvýbor.*)', re.U | re.I)

	for g in Group.objects.all():
		g.name = g.origName.strip()
		g.type = g.type.strip().upper()
		g.save()

	for g in Group.objects.filter(origName__iregex = u'podvýbor'):
		match = splitPattern.match(g.origName)
		if match != None:
			boardName = match.group('boardName').strip()
			g.name = match.group('subBoardName').strip()

			try:
				pg = Group.objects.filter(origName__iexact = boardName, term = g.term)[0]
				g.parentGroup = pg
				print '%s -> %s' % (g.origName, g.name)
			except:
				pass

			# force presave commit hook to regenerate proper slug
			g.save()

	for g in Group.objects.all():
		g.slug = ''
		g.save()

def makeName(str):
	"""
	It gets string in form 'Josef Karel Novak' the first word 'Josef' is
	stripped and it is returned as a name, the rest of the string is returned
	as a surname.
	"""
	name = ''
	surname = ''
	s = str.split()
	name = s[0]
	for i in s[1:]:
		surname += i + ' '

	return name.strip(), surname.strip()
		

def getGroup(orgId, term):
	"""
	gets group object for given ID and term ID. if no group object exists, it
	is fetched from psp.cz and new record is created in DB.
	"""
	retVal = None
	if len(orgId) > 0:
		try:
			retVal = Group.objects.get(groupId__exact = orgId, term = term) 
			print 'Known Group: %s (%s)' % (retVal.origName, retVal.groupId)
		except ObjectDoesNotExist:
			groupName = utils.fetcher.getOrgName(orgId, term.termId)
			print 'New Group: %s (%s) %s' % (groupName, orgId, term.termId)
			retVal = Group()
			retVal.term = term
			retVal.origName = groupName.strip()
			retVal.groupId = orgId.strip()
			retVal.save()

	return retVal

	
def getArea(areaId, term):
	"""
	gets Area name for given area ID and term ID if area does not exists in DB,
	it is fetched from psp.cz site and new record is created in DB
	"""
	retVal = None
	try:
		retVal = Division.objects.get(divId__exact = areaId, term = term)
		print 'Known Area: %s (%s)' % (retVal.name, retVal.divId)
	except ObjectDoesNotExist:
		retVal = Division()
		divName = utils.fetcher.getOrgName(areaId, term.termId)
		print 'New Area: %s (%s)' % (divName, areaId)
		retVal.term = term
		retVal.name = divName.strip()
		retVal.divId = areaId.strip()
		retVal.save()
		
	return retVal


def updatePersonObject(person, details):
	"""
	updates object from details, which are
	fetched from psp.cz site
	"""
	if len(details['homePage']) > 0:
		person.homePage = details['homePage'].strip()

	if len(details['regOffice']) > 0:
		person.regOffice = details['regOffice'].strip()

	if len(details['office']) > 0:
		person.office = details['office'].strip()

	if len(details['phone']) > 0:
		person.phone = details['phone'].strip()

	if len(details['email']) > 0:
		person.email = details['email'].strip()

	person.save()
	

def updateDetail(person, term, area, mpDetails):
	"""
	creates detail object for person if it does not exist.
	person,	term, area are django objects,
	mpDetails is dictionary 
	"""
	try:
		detail = PersonDetail.objects.get(person = person, term = term, region = area)
		print 'Detail for %s %s is up to date' % (person.name, person.surname)
	except ObjectDoesNotExist:
		detail = PersonDetail()

		try:
			f = open(mpDetails['photoFile'], 'r')
			detail.photo.save(mpDetails['photoFile'], File(f), save = True)
		except:
			pass

		detail.term = term
		detail.person = person
		detail.region = area
		detail.save()
		print 'Created new detail record for %s %s' % (person.name, person.surname)


def updateGroups(term, mpDict):
	"""
	creates parliamentary groups, which does not exist yet.  the termId is
	string with termId, mpDict are data from list of MPs found
	http://www.psp.cz/sqw/snem.sqw?P1=0&P2=0&l=cz
	"""
	for b in mpDict['boardIds']:
		board = getGroup(b, term)
		if board != None:
			board.save()

	for c in mpDict['commIds']:
		committee = getGroup(c, term)
		if committee != None:
			committee.save()

	for d in mpDict['delegIds']:
		deleg = getGroup(d, term)
		if deleg != None:
			deleg.save()


def updateMemberships(person, term, memberships):
	"""
	Creates membership record objects.
	"""
	for m in memberships:
		g = getGroup(m['orgId'], term)
		if g == None:
			try:
				g = Group.objects.get(origName__iexact = m['orgName'], term = term)
			except ObjectDoesNotExist:
				print 'No group found for %s (%s)' % (m['orgName'], m['orgId'])
				g = Group()
				g.term = term
				g.origName = m['orgName']
				g.name = m['orgName']
				g.groupId = m['orgId']
				g.type = m['orgType']

		g.type = m['orgType']
		if len(g.origName) == 0:
			g.origName = m['orgName']
		g.save()

		start = datetime.datetime(
				year = m['start']['year'],
				month = m['start']['month'],
				day = m['start']['day']
			)
		membership = Membership.objects.filter(
							group = g,
							person = person,
							post__iexact = m['post'],
							start = start
						)
		if len(membership) != 0:
			if len(membership) == 1:
				try:
					end = datetime.datetime(
							year = m['end']['year'],
							month = m['end']['month'],
							day = m['end']['day']
						)
					if membership[0].end != end:
						membership[0].end = end
						membership[0].save()
				except:
					pass
			else:
				print 'Warning found more identical memberships for %s %s (%s)' % (
					person.name,
					person.surname,
					person.mpId
				)
				for i in membership:
					print '\t%s - %s - (%d %d %d - %d %d %d)' % (
						i.group.origName,
						i.post,
						i.start.day,
						i.start.month,
						i.start.year,
						i.end.day,
						i.end.month,
						i.end.year
					)

		else:
			membership = Membership()
			if len(g.type) == 0:
				g.type = 'PARL_GROUP'
			membership.group = g
			membership.person = person
			membership.post = m['post']
			membership.start = start
			try:
				membership.end = datetime.datetime(
									year = m['end']['year'],
									month = m['end']['month'],
									day = m['end']['day']
								)
			except:
				pass
			membership.save()

			
def updateMPs(mps, term):
	"""
	It gets list of MPs fetched from
		http://www.psp.cz/sqw/snem.sqw?P1=0&P2=0&l=cz
	it walks through list and updates details for every MP
	"""

	for mp in mps:
		p = None
		personRecord = utils.fetcher.fetchMemberByID(mp['pspId'].strip())
		try:
			p = Person.objects.get(mpId__exact = mp['pspId'].strip())
			print 'Updating: %s %s (%s)' % (p.name, p.surname, p.mpId)
		except ObjectDoesNotExist:
			p = Person()
			p.name, p.surname = makeName(mp['name'])
			p.title = mp['titlePre']
			p.titleLast = mp['titlePost']
			p.birthDate = datetime.date(
							year = personRecord['birthDate']['year'],
							month = personRecord['birthDate']['month'],
							day = personRecord['birthDate']['day']
						)
			p.mpId = mp['pspId'].strip()
			p.save()
			print 'Creating: %s %s (%s)' % (p.name, p.surname, p.mpId)

		party = getGroup(mp['partyId'], term)
		party.type = 'PARTY'
		party.save()
		#
		# this will allow us to postprocess groups later (see fixGroups())
		# once fixing names will be done in code, we can remove it
		#
		updateGroups(term, mp)

		mpDetails = utils.fetcher.fetchMP(mp['pspId'], term.termId)
		if mpDetails != None:
			updatePersonObject(p, mpDetails)
			area = getArea(mp['areaId'], term)
			updateDetail(p, term, area, mpDetails)
			updateMemberships(p, term, mpDetails['membship'])


def updateMeetings(meetings, term):
	for m in meetings:
		try:
			meeting = Meeting.objects.get(
							term = term,
							meetingNo__iexact = m['meetingNo']
						)
		except ObjectDoesNotExist:
			meeting = Meeting()
			meeting.meetingNo = m['meetingNo']
			meeting.name = m['name']
			meeting.url = 'http://www.psp.cz/sqw%s' % (m['url'])
			start = m['days'][0]
			end = m['days'][len( m['days']) - 1 ]
			meeting.start = datetime.datetime(
									year = start['year'],
									month = start['month'],
									day = start['day']
								)
			meeting.end = datetime.datetime(
									year = end['year'],
									month = end['month'],
									day = end['day']
								)
			meeting.term = term
			meeting.save()
			print 'Saving meeting: %s (%s)' % (meeting.name, term.termId)



def updateStats(term, result):
	"""
	function updates stats for MP in given date range
	"""
	mpStat = None
	try:
		mpStat = VoteStats.objects.get(
					person = result.person,
					term = term
				)
	except ObjectDoesNotExist:
		print 'Creating stat for %s %s' % (result.person.name, result.person.surname)
		mpStat = VoteStats()
		mpStat.person = result.person
		mpStat.term = term
		mpStat.votesAye = 0
		mpStat.votesNay = 0
		mpStat.refrains = 0
		mpStat.absences = 0
		mpStat.excused = 0
		

	if result.result == 'A':
		mpStat.votesAye += 1
	elif result.result == 'N':
		mpStat.votesNay += 1
	elif result.result == '0':
		mpStat.absences += 1
	elif result.result == 'Z':
		mpStat.refrains += 1
	elif result.result == 'M' or result.result == 'X':
		mpStat.excused += 1
	mpStat.save()


def updatePolls(term):
	for m in Meeting.objects.filter(term = term):
		results = utils.fetcher.fetchResultsOverview(term.termId, m.meetingNo)
		for r in results:
			poll = None
			try:
				poll = Poll.objects.get(meeting = m, pollId__iexact = r['pollId'])
				print 'Poll %s already exists' % poll.name
			except ObjectDoesNotExist:
				poll = Poll()
				poll.meeting = m
				poll.pollId = r['pollId']
				poll.name = r['pollName']
				poll.verdict = r['pollResult']
				poll.resultsLink = r['pollUrl']
				poll.stenoLink = r['pollDetailsURL']
				poll.date = datetime.datetime(
								year = r['pollDate']['year'],
								month = r['pollDate']['month'],
								day = r['pollDate']['day']
							)
				poll.votesAye = 0
				poll.votesNay = 0
				poll.votesRefrain = 0
				poll.absences = 0
				poll.excused = 0
				print 'Saved poll: %s' % (poll.name,)
				poll.save()

			if poll.votesAye == 0 and\
				poll.votesNay == 0 and\
				poll.votesRefrain == 0 and\
				poll.excused == 0 and\
				poll.absences == 0:

				votes = utils.fetcher.fetchPollResult(term.termId, poll.pollId)

				for vote in votes:
					person = None
					try:
						person = Person.objects.get(mpId__iexact = vote['pspId'])
					except ObjectDoesNotExist:
						continue

					v = MpVote()
					v.poll = poll
					v.person = person
					v.result = vote['result']
					try:
						v.save()
					except IntegrityError:
						# the vote for MP already exists, ignore it
						continue
					poll.bumpPoll(v.result)
						
				poll.save()
				GroupStats.objects.createStats(poll)

			else:
				print 'Results already exists for %s' % (poll.name,)


def updateDataForTerm(term = None):
	"""
	updates data for given term. if no term is passed
	it processes all terms.
	"""
	if term == None:
		try:
			terms = TermOfOffice.objects.all().order_by('id')
		except:
			print 'No terms are defined'
			return
		
		for term in terms:
			mps = utils.fetcher.fetchPSPMembers(term.termId)
			updateMPs(mps, term)
			meetings = utils.fetcher.fetchPSPMeetings(term.termId)
			updateMeetings(meetings, term)
			updatePolls(term)
	else:
		print 'Updating term: %s - %s'  % (term.yearStart, term.yearEnd)
		mps = utils.fetcher.fetchPSPMembers(term.termId)
		updateMPs(mps, term)
		meetings = utils.fetcher.fetchPSPMeetings(term.termId)
		updateMeetings(meetings, term)
		updatePolls(term)


def update():
	lastTerm = None

	terms = TermOfOffice.objects.all().order_by('id')
	if len(terms) > 0:
		lastTerm = terms[len(terms) - 1]

	webTerms = utils.fetcher.fetchTerms()
	webTerms.reverse()

	if lastTerm == None:
		# DB is empty
		for webTerm in webTerms:
			term = TermOfOffice()
			term.yearStart = webTerm['yearStart']
			term.yearEnd = webTerm['yearEnd']
			term.termId = webTerm['termId']
			term.save()
			print 'Created Term: %s' % term.slug
			updateDataForTerm(term)
	else:
		# Find last term in DB to start update
		i = 0
		for webTerm in webTerms:
			if webTerm['termId'] == lastTerm.termId:
				break
			i += 1

		print "Starting at %s - %s" % (lastTerm.yearStart, lastTerm.yearEnd)
		updateDataForTerm(lastTerm)
		i += 1

		for webTerm in webTerms[i:]:
			term = TermOfOffice()
			term.yearStart = webTerm['yearStart']
			term.yearEnd = webTerm['yearEnd']
			term.termId = webTerm['termId']
			term.save()
			print 'Created Term: %s' % term.slug
			updateDataForTerm(term)


def fetchMissingResults():
	def getMissingResults():
		retVal = []
		for p in Poll.objects.filter(
								vote_aye = 0,
								vote_nay = 0,
								refrain = 0,
								absentees = 0,
								excused = 0
							):
			if MpVote.objects.filter(poll = p).count() == 0:
				retVal.append(p)

		return retVal

	def savePollResults(poll):
		votes = utils.fetcher.fetchPollResult(poll.meeting.term.termId, poll.pollId)
		term = poll.meeting.term
		print 'Fetched votes for %s' % (poll.name)
		for vote in votes:
			person = None
			try:
				person = Person.objects.get(mpId__iexact = vote['pspId'])
			except ObjectDoesNotExist:
				continue

			"""
			We are not going to look up single vote here avoiding creating
			duplicate votes here. we are going to save all or none.
			"""
			v = MpVote()
			v.poll = poll
			v.person = person
			v.result = vote['result']
			try:
				v.save()
			except IntegrityError:
				# the vote for MP already exists, ignore it
				continue

			poll.bumpPoll(v.result)					
			poll.save()

	for p in getMissingResults():
		savePollResults(p)
		GroupStats.objects.createStats(poll)
	
def main():
	update()
	fixNames()
	fixOrgs()
	#fetchMissingResults()
#	meetings = Meeting.objects.filter(meetingNo__exact = '53')
#	for m in meetings:
#		print (m.start, ' - ', m.end)
#	
#	m = meetings[0]
#	polls = Poll.objects.filter(meeting = m)
#	
#	g = Group.objects.get(groupId__exact = '-1')
#	for p in polls:
#		members = Membership.objects.filter(start__gte='2002-01-01', end__lte='2006-06-30', group = g)
#		for member in members:
#			votes = MpVote.objects.filter(poll = p, person = member.person)
#			if votes:
#				if len(votes) > 1:
#					print p.name
#					delThem = {}
#
#					r = votes[0].result
#					doDelete = True
#					for v in votes:
#						print '%d: %s %s' % (v.id, member.person.name, member.person.surname)
#						delThem[v.id] = v
#						if v.result != r:
#							doDelete = False
#
#					if doDelete:
#						print 'going to normalize result'
#						list = delThem.keys()
#						list.sort()
#						for i in list[0:len(list) - 1]:
#							print 'deleting: %d' % delThem[i].id
#							delThem[i].delete()

if __name__ == '__main__':
	main()
	sys.exit()
